{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d1cefa5d",
   "metadata": {},
   "source": [
    "Q(state,action) = state下最优action分数 + 误差\n",
    "\n",
    "为了限制误差的范围,可以对它去均值,相当于去基线\n",
    "\n",
    "Q(state,action) = state下最优action分数 + 误差 - mean(误差)\n",
    "\n",
    "这么理解起来有点蛋疼,所以我这么理解\n",
    "\n",
    "Q(state,action) = state分数 + action分数 - mean(action分数)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUXUlEQVR4nO3df0wb990H8LcN2JDAmQDFLgtemFItQ/mxjSTk2j82LW5Yh6ZlRXq2KepYFaVqZqKmTJGG1NKlTx/RJ/2jW7aU/NEtqfSoY2NSNhWlzcNDEqIpbkhokQhJUKt1grWx3SbjDARs4/s8f0Tc6oZ0OBB/cfx+SSfl7j7GnzvwO+f7ns82EREQESlgV90AEWUvBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESmjLIAOHTqEVatWIT8/H7W1tejr61PVChEpoiSA/vCHP6C5uRnPPfcc3nnnHWzYsAF1dXUIh8Mq2iEiRWwqPoxaW1uLTZs24Te/+Q0AwDRNVFZWYs+ePfj5z3+e7naISJHcdD9hLBZDf38/WlparGV2ux0+nw+BQGDOx0SjUUSjUWveNE1cv34dpaWlsNlsd71nIkqNiGB8fBwVFRWw22//RivtAfTJJ58gkUjA7XYnLXe73bhy5cqcj2lra8P+/fvT0R4RLaLR0VGsXLnytuvTHkB3oqWlBc3Nzda8YRjwer0YHR2FpmkKOyOiuUQiEVRWVqKoqOhz69IeQGVlZcjJyUEoFEpaHgqF4PF45nyM0+mE0+m8ZbmmaQwgoiXs350iSfsomMPhQE1NDXp6eqxlpmmip6cHuq6nux0iUkjJW7Dm5mY0NjZi48aN2Lx5M375y19icnISjz/+uIp2iEgRJQH0gx/8AB9//DFaW1sRDAbx1a9+FW+99dYtJ6aJ6N6m5DqghYpEInC5XDAMg+eAiJag+b5G+VkwIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKZNyAJ05cwbf/e53UVFRAZvNhj//+c9J60UEra2tuP/++1FQUACfz4f33nsvqeb69evYsWMHNE1DcXExdu7ciYmJiQVtCBFlnpQDaHJyEhs2bMChQ4fmXH/gwAEcPHgQhw8fxrlz57B8+XLU1dVhenraqtmxYweGhobQ3d2Nrq4unDlzBk888cSdbwURZSZZAABy7Ngxa940TfF4PPLSSy9Zy8bGxsTpdMrvf/97ERG5dOmSAJDz589bNW+++abYbDb58MMP5/W8hmEIADEMYyHtE9FdMt/X6KKeA/rggw8QDAbh8/msZS6XC7W1tQgEAgCAQCCA4uJibNy40arx+Xyw2+04d+7cnD83Go0iEokkTUSU+RY1gILBIADA7XYnLXe73da6YDCI8vLypPW5ubkoKSmxaj6rra0NLpfLmiorKxezbSJSJCNGwVpaWmAYhjWNjo6qbomIFsGiBpDH4wEAhEKhpOWhUMha5/F4EA6Hk9bPzMzg+vXrVs1nOZ1OaJqWNBFR5lvUAKqqqoLH40FPT4+1LBKJ4Ny5c9B1HQCg6zrGxsbQ399v1Zw8eRKmaaK2tnYx2yGiJS431QdMTEzg/ffft+Y/+OADDAwMoKSkBF6vF3v37sULL7yABx54AFVVVXj22WdRUVGB7du3AwC+8pWv4Nvf/jZ27dqFw4cPIx6Po6mpCT/84Q9RUVGxaBtGRBkg1eG1U6dOCYBbpsbGRhG5ORT/7LPPitvtFqfTKVu3bpXh4eGkn3Ht2jX50Y9+JIWFhaJpmjz++OMyPj6+6EN8RKTGfF+jNhERhfl3RyKRCFwuFwzD4PkgoiVovq/RjBgFI6J7EwOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUiblr+UhWizxqXFE/nHJmrfZc1D8xfWw5zoUdkXpxAAiZaau/wN/O/lbaz7HUYC1//ELOBhAWYNvwUiZRDymugVSjAFEypgzDKBsxwAiZcwEAyjbMYBImfikkTRvz3PCZstR1A2pwAAiZaKRj5PmHYUlsOc5FXVDKjCAaMmw5+TBZrOpboPSKKUAamtrw6ZNm1BUVITy8nJs374dw8PDSTXT09Pw+/0oLS1FYWEhGhoaEAqFkmpGRkZQX1+PZcuWoby8HPv27cPMzMzCt4Yymi0nF7Dx/8RsktJvu7e3F36/H2+//Ta6u7sRj8exbds2TE5OWjVPP/003njjDXR2dqK3txcfffQRHn30UWt9IpFAfX09YrEYzp49i9deew1Hjx5Fa2vr4m0VZaSbR0AMoKwiCxAOhwWA9Pb2iojI2NiY5OXlSWdnp1Vz+fJlASCBQEBERI4fPy52u12CwaBV097eLpqmSTQandfzGoYhAMQwjIW0TwqZpil/O3VE+g7vsqb3/vewmKapujVaBPN9jS7ovxvDuDmKUVJSAgDo7+9HPB6Hz+ezatasWQOv14tAIAAACAQCWLduHdxut1VTV1eHSCSCoaGhOZ8nGo0iEokkTZTpBGYi+W23zWbjOaAsc8cBZJom9u7di4ceeghr164FAASDQTgcDhQXFyfVut1uBINBq+bT4TO7fnbdXNra2uByuaypsrLyTtumpUIEZiKuugtS7I4DyO/34+LFi+jo6FjMfubU0tICwzCsaXR09K4/J91dIiaEAZT17ujDqE1NTejq6sKZM2ewcuVKa7nH40EsFsPY2FjSUVAoFILH47Fq+vr6kn7e7CjZbM1nOZ1OOJ28PuSeIgKTI59ZL6UjIBFBU1MTjh07hpMnT6KqqippfU1NDfLy8tDT02MtGx4exsjICHRdBwDouo7BwUGEw2Grpru7G5qmobq6eiHbQhnEnIkjNnEtaZmjqExRN6RKSkdAfr8fr7/+Ov7yl7+gqKjIOmfjcrlQUFAAl8uFnTt3orm5GSUlJdA0DXv27IGu69iyZQsAYNu2baiursZjjz2GAwcOIBgM4plnnoHf7+dRThYRSSARm0pa5li+QlE3pEpKAdTe3g4A+OY3v5m0/MiRI/jJT34CAHj55Zdht9vR0NCAaDSKuro6vPLKK1ZtTk4Ourq6sHv3bui6juXLl6OxsRHPP//8wraEMh5vRJZ9bCIiqptIVSQSgcvlgmEY0DRNdTt0B+JTEVz84y8wMz1hLfvS1l0oXb1JYVe0WOb7GuVlp7Rk8Ago+zCASIm5DrztObxDcLZhAJESMhOHiKm6DVKMAURKmIk4kHmnH2mRMYBICTMRn/NtGGUXBhApYc7EgM++BeMHUbMOA4iUiBrhpG/FyHUuh6OwRGFHpAIDiJQQM5E0b8vJhT0nT1E3pAoDiJYEm81+85aslFUYQLQ02HN4HVAWYgBR2s01+mWz22GzM4CyDQOIlDA/ew4IvB1rNmIAkRLmTFR1C7QEMIBIiU8PwVP2YgCREuYM7wdNDCBSZGYq+auV7Hn5AHgOKNswgEgBwfRY8td1F6zwwGbnn2O24W+clgRbjgM8Aso+DCBaEuy5ecyfLMQAoiXh5u1YmUDZhgFEiiRfDc0PomYnBhClnZgmxEy+F5DNxiuhsxEDiNJOzARMk1/LTAwgUkDMxC33A6LsxI8f06ITEYyPj8M05/7Wi0R0Eol48pXQ09PTGBsbm7PebrejqKiIb9HuQQwgWnQTExN4+OGH8eGHH865/gulhTiw80EUOG7++ZliovU//xunL+6ds3716tU4ceIEnE7n3WqZFGEA0aITEQSDwdsG0DJbCW6IG5fHa2CDiS86zuP9kau3rS8qKuI3aNyjUjoH1N7ejvXr10PTNGiaBl3X8eabb1rrp6en4ff7UVpaisLCQjQ0NCAUSr7kfmRkBPX19Vi2bBnKy8uxb98+zMzwhGQ2mTKLMDDuQzi2CqHYl/BuZCsiUQ7DZ6OUAmjlypV48cUX0d/fjwsXLuBb3/oWvve972FoaAgA8PTTT+ONN95AZ2cnent78dFHH+HRRx+1Hp9IJFBfX49YLIazZ8/itddew9GjR9Ha2rq4W0VL2ozpwLS53JqfShRiMsrxkKwkC7RixQp59dVXZWxsTPLy8qSzs9Nad/nyZQEggUBARESOHz8udrtdgsGgVdPe3i6apkk0Gp33cxqGIQDEMIyFtk93gWEY4vV6BTevNrxlqvrC/XKw7Y+y/4Vzsv+Fc3Lwv/5HvJ77blu/Zs0amZqaUr1ZlIL5vkbv+BxQIpFAZ2cnJicnoes6+vv7EY/H4fP5rJo1a9bA6/UiEAhgy5YtCAQCWLduHdxut1VTV1eH3bt3Y2hoCF/72tdS6uHKlSsoLCy8002gu2RychLx+O3v9zMW+SfO9ryEcLwKNpugLOdv+GfEuG19NBrFlStX4HA47ka7dBdMTEzMqy7lABocHISu65ienkZhYSGOHTuG6upqDAwMwOFwoLi4OKne7XYjGAwCAILBYFL4zK6fXXc70WgU0ei/buEZidy8l4xhGDx/tATduHHjtkPwAPDP8Wl0/N95AOfn9fNM04RhGMjL43miTDE5OTmvupQD6Mtf/jIGBgZgGAb+9Kc/obGxEb29vSk3mIq2tjbs37//luW1tbXQNO2uPjelLhKJLOqQeUFBAWpra5Gfn79oP5PurtmDhH8n5TN/DocDq1evRk1NDdra2rBhwwb86le/gsfjQSwWu+VislAoBI/HAwDweDy3jIrNzs/WzKWlpQWGYVjT6Ohoqm0T0RK04KEH0zQRjUZRU1ODvLw89PT0WOuGh4cxMjICXdcBALquY3BwEOFw2Krp7u6Gpmmorq6+7XM4nU5r6H92IqLMl9JbsJaWFjzyyCPwer0YHx/H66+/jtOnT+PEiRNwuVzYuXMnmpubUVJSAk3TsGfPHui6ji1btgAAtm3bhurqajz22GM4cOAAgsEgnnnmGfj9fl7lSpSFUgqgcDiMH//4x7h69SpcLhfWr1+PEydO4OGHHwYAvPzyy7Db7WhoaEA0GkVdXR1eeeUV6/E5OTno6urC7t27oes6li9fjsbGRjz//POLu1WklM1mQ1FR0aIdqXKk895lE8m8a9wjkQhcLhcMw+DbsSXINE0Eg0EkEovziffc3Fx4PB5+GDWDzPc1ys+C0aKz2+2oqKhQ3QZlAF7/TkTKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJTJVd3AnRARAEAkElHcCRHNZfa1OftavZ2MDKBr164BACorKxV3QkSfZ3x8HC6X67brMzKASkpKAAAjIyOfu3GULBKJoLKyEqOjo9A0TXU7GYH77M6ICMbHx1FRUfG5dRkZQHb7zVNXLpeLfxR3QNM07rcUcZ+lbj4HBzwJTUTKMICISJmMDCCn04nnnnsOTqdTdSsZhfstddxnd5dN/t04GRHRXZKRR0BEdG9gABGRMgwgIlKGAUREymRkAB06dAirVq1Cfn4+amtr0dfXp7olZdra2rBp0yYUFRWhvLwc27dvx/DwcFLN9PQ0/H4/SktLUVhYiIaGBoRCoaSakZER1NfXY9myZSgvL8e+ffswMzOTzk1R5sUXX4TNZsPevXutZdxnaSIZpqOjQxwOh/zud7+ToaEh2bVrlxQXF0soFFLdmhJ1dXVy5MgRuXjxogwMDMh3vvMd8Xq9MjExYdU8+eSTUllZKT09PXLhwgXZsmWLPPjgg9b6mZkZWbt2rfh8Pnn33Xfl+PHjUlZWJi0tLSo2Ka36+vpk1apVsn79ennqqaes5dxn6ZFxAbR582bx+/3WfCKRkIqKCmlra1PY1dIRDocFgPT29oqIyNjYmOTl5UlnZ6dVc/nyZQEggUBARESOHz8udrtdgsGgVdPe3i6apkk0Gk3vBqTR+Pi4PPDAA9Ld3S3f+MY3rADiPkufjHoLFovF0N/fD5/PZy2z2+3w+XwIBAIKO1s6DMMA8K8P7Pb39yMejyftszVr1sDr9Vr7LBAIYN26dXC73VZNXV0dIpEIhoaG0th9evn9ftTX1yftG4D7LJ0y6sOon3zyCRKJRNIvHQDcbjeuXLmiqKulwzRN7N27Fw899BDWrl0LAAgGg3A4HCguLk6qdbvdCAaDVs1c+3R23b2oo6MD77zzDs6fP3/LOu6z9MmoAKLP5/f7cfHiRfz1r39V3cqSNjo6iqeeegrd3d3Iz89X3U5Wy6i3YGVlZcjJybllNCIUCsHj8SjqamloampCV1cXTp06hZUrV1rLPR4PYrEYxsbGkuo/vc88Hs+c+3R23b2mv78f4XAYX//615Gbm4vc3Fz09vbi4MGDyM3Nhdvt5j5Lk4wKIIfDgZqaGvT09FjLTNNET08PdF1X2Jk6IoKmpiYcO3YMJ0+eRFVVVdL6mpoa5OXlJe2z4eFhjIyMWPtM13UMDg4iHA5bNd3d3dA0DdXV1enZkDTaunUrBgcHMTAwYE0bN27Ejh07rH9zn6WJ6rPgqero6BCn0ylHjx6VS5cuyRNPPCHFxcVJoxHZZPfu3eJyueT06dNy9epVa7px44ZV8+STT4rX65WTJ0/KhQsXRNd10XXdWj87pLxt2zYZGBiQt956S+67776sGlL+9CiYCPdZumRcAImI/PrXvxav1ysOh0M2b94sb7/9tuqWlAEw53TkyBGrZmpqSn7605/KihUrZNmyZfL9739frl69mvRz/v73v8sjjzwiBQUFUlZWJj/72c8kHo+neWvU+WwAcZ+lB2/HQUTKZNQ5ICK6tzCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZf4f+8HTf6LGgikAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "378392ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 2])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class Model(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.fc = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 64),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        self.fc_action = torch.nn.Linear(64, 2)\n",
    "        self.fc_state = torch.nn.Linear(64, 1)\n",
    "\n",
    "    def forward(self, state):\n",
    "        state = self.fc(state)\n",
    "\n",
    "        #评估state的价值\n",
    "        value_state = self.fc_state(state)\n",
    "\n",
    "        #每个state下每个action的价值\n",
    "        value_action = self.fc_action(state)\n",
    "\n",
    "        #综合以上两者计算最终的价值,action去均值是为了数值稳定\n",
    "        return value_state + value_action - value_action.mean(dim=-1,\n",
    "                                                              keepdim=True)\n",
    "\n",
    "\n",
    "model = Model()\n",
    "model_delay = Model()\n",
    "\n",
    "#复制参数\n",
    "model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "model(torch.randn(64, 4)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\appDir\\python3.10\\lib\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  if not isinstance(terminated, (bool, np.bool8)):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-989.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "        if random.random() < 0.1:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bfcfd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_13868\\2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(221,\n",
       " (array([ 0.00472838, -0.02780635,  0.00946377, -0.04235087], dtype=float32),\n",
       "  1,\n",
       "  1.0,\n",
       "  array([ 0.00417226,  0.16717863,  0.00861676, -0.33203292], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 447 -991.1\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(100):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                target = model_delay(next_state)\n",
    "            target = target.max(dim=1)[0].reshape(-1, 1)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        #复制参数\n",
    "        if (epoch + 1) % 5 == 0:\n",
    "            model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAASyklEQVR4nO3df0xTd78H8DcFWkU4ZeBox5U+kozNEX9tqHj0jy3awZSYOfljW4xjxmh0xagsJuNef0S3BeOSubkx/GMbmuxxLCxxi8QfI6CYxSKK414E5W5P3IWobadeTgFHC/R7/3gu51knOorYL5X3KzmJ5/v99PRzvtq37TkWo4QQAkREEhhkN0BE4xcDiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpJEWQKWlpZg6dSomTJiA7OxsNDQ0yGqFiCSREkDffPMNioqKsHPnTly8eBGzZs1Cbm4uPB6PjHaISJIoGV9Gzc7Oxty5c/Hpp58CAAKBANLS0rBx40a888474W6HiCSJCfcT+v1+NDY2ori4WB8zGAyw2+1wOp1DPsbn88Hn8+n7gUAAt2/fRnJyMqKioh56z0QUGiEEurq6kJqaCoPh3h+0wh5AN2/exMDAACwWS9C4xWLBlStXhnxMSUkJdu3aFY72iGgUdXR0YMqUKfecD3sAjURxcTGKior0fU3TYLPZ0NHRAUVRJHZGREPxer1IS0tDQkLCfevCHkCTJ09GdHQ03G530Ljb7YbVah3yMSaTCSaT6a5xRVEYQERj2F9dIgn7XTCj0YisrCzU1NToY4FAADU1NVBVNdztEJFEUj6CFRUVoaCgAHPmzMG8efPw0UcfoaenB6tXr5bRDhFJIiWAXn31Vfz222/YsWMHXC4XZs+ejRMnTtx1YZqIHm1S/h3Qg/J6vTCbzdA0jdeAiMag4b5G+V0wIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARSRNyAJ05cwbLli1DamoqoqKi8N133wXNCyGwY8cOPPHEE5g4cSLsdjt+/vnnoJrbt29j5cqVUBQFiYmJWLNmDbq7ux/oRIgo8oQcQD09PZg1axZKS0uHnN+7dy/279+PAwcO4Ny5c5g0aRJyc3PR29ur16xcuRItLS2orq5GVVUVzpw5g3Xr1o38LIgoMokHAEAcOXJE3w8EAsJqtYoPPvhAH+vs7BQmk0l8/fXXQgghWltbBQBx/vx5veb48eMiKipKXLt2bVjPq2maACA0TXuQ9onoIRnua3RUrwFdvXoVLpcLdrtdHzObzcjOzobT6QQAOJ1OJCYmYs6cOXqN3W6HwWDAuXPnhjyuz+eD1+sN2ogo8o1qALlcLgCAxWIJGrdYLPqcy+VCSkpK0HxMTAySkpL0mj8rKSmB2WzWt7S0tNFsm4gkiYi7YMXFxdA0Td86Ojpkt0REo2BUA8hqtQIA3G530Ljb7dbnrFYrPB5P0Hx/fz9u376t1/yZyWSCoihBGxFFvlENoPT0dFitVtTU1OhjXq8X586dg6qqAABVVdHZ2YnGxka9pra2FoFAANnZ2aPZDhGNcTGhPqC7uxu//PKLvn/16lU0NTUhKSkJNpsNmzdvxnvvvYeMjAykp6dj+/btSE1NxfLlywEAzzzzDF566SWsXbsWBw4cQF9fHwoLC/Haa68hNTV11E6MiCJAqLfXTp06JQDctRUUFAgh/nkrfvv27cJisQiTySQWL14s2trago5x69Yt8frrr4v4+HihKIpYvXq16OrqGvVbfEQkx3Bfo1FCCCEx/0bE6/XCbDZD0zReDyIag4b7Go2Iu2BE9GhiABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETShPzf8hCNdUIIdN34b/T1dOpjExKtmPT43+Q1RUNiANEjyfWf1dDa/0vfT5m+iAE0BvEjGBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpImpAAqKSnB3LlzkZCQgJSUFCxfvhxtbW1BNb29vXA4HEhOTkZ8fDzy8/PhdruDatrb25GXl4e4uDikpKRg69at6O/vf/CzIaKIElIA1dXVweFwoL6+HtXV1ejr60NOTg56enr0mi1btuDo0aOorKxEXV0drl+/jhUrVujzAwMDyMvLg9/vx9mzZ3Ho0CEcPHgQO3bsGL2zIqLIIB6Ax+MRAERdXZ0QQojOzk4RGxsrKisr9ZrLly8LAMLpdAohhDh27JgwGAzC5XLpNWVlZUJRFOHz+Yb1vJqmCQBC07QHaZ8eUYFAQLQd+0Q0HFirb7/++LXstsaV4b5GH+gakKZpAICkpCQAQGNjI/r6+mC32/WaadOmwWazwel0AgCcTidmzJgBi8Wi1+Tm5sLr9aKlpWXI5/H5fPB6vUEbEUW+EQdQIBDA5s2bsXDhQkyfPh0A4HK5YDQakZiYGFRrsVjgcrn0mj+Gz+D84NxQSkpKYDab9S0tLW2kbRPRGDLiAHI4HLh06RIqKipGs58hFRcXQ9M0fevo6Hjoz0lED9+IvoxaWFiIqqoqnDlzBlOmTNHHrVYr/H4/Ojs7g94Fud1uWK1WvaahoSHoeIN3yQZr/sxkMsFkMo2kVSIaw0J6BySEQGFhIY4cOYLa2lqkp6cHzWdlZSE2NhY1NTX6WFtbG9rb26GqKgBAVVU0NzfD4/HoNdXV1VAUBZmZmQ9yLkQUYUJ6B+RwOHD48GF8//33SEhI0K/ZmM1mTJw4EWazGWvWrEFRURGSkpKgKAo2btwIVVUxf/58AEBOTg4yMzOxatUq7N27Fy6XC9u2bYPD4eC7HKJxJqQAKisrAwC88MILQePl5eV48803AQD79u2DwWBAfn4+fD4fcnNz8dlnn+m10dHRqKqqwoYNG6CqKiZNmoSCggLs3r37wc6EiCJOlBBCyG4iVF6vF2azGZqmQVEU2e3QGCOEwM8nSu/6gWR/W/iaxK7Gl+G+RvldMCKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQh/c+oRGNFb28vent77znf398XtO/3+dDZ2XnP+ri4OBiNxtFqj4aJAUQR6fPPP8eePXvuOf/vr2ZhToZF3//q73/HF2/8xz3ry8rKsGzZslHtkf4aA4giUldXF65du3bP+Z7fZ+PnO1n4334rLMb/QVdX633r79y58zDapL8Q0jWgsrIyzJw5E4qiQFEUqKqK48eP6/O9vb1wOBxITk5GfHw88vPz4Xa7g47R3t6OvLw8xMXFISUlBVu3bkV/f//onA3R//vlznP4x+/P4Xbfv+FKTzY6fNNkt0RDCCmApkyZgj179qCxsREXLlzAokWL8PLLL6OlpQUAsGXLFhw9ehSVlZWoq6vD9evXsWLFCv3xAwMDyMvLg9/vx9mzZ3Ho0CEcPHgQO3bsGN2zonGve+AxAFEAAIFo9AwkSu2HhhZSAC1btgxLly5FRkYGnnrqKbz//vuIj49HfX09NE3DF198gQ8//BCLFi1CVlYWysvLcfbsWdTX1wMAfvjhB7S2tuKrr77C7NmzsWTJErz77rsoLS2F3+9/KCdI49MTpl8QE+UDIGCKugOL8VfZLdEQRnwNaGBgAJWVlejp6YGqqmhsbERfXx/sdrteM23aNNhsNjidTsyfPx9OpxMzZsyAxfKvi4O5ubnYsGEDWlpa8Oyzz4bUw5UrVxAfHz/SU6AI5vF47jv/08XjmHT1H9D6H0dS7A1cu3blvvXXrl1Da2vraLY4rnV3dw+rLuQAam5uhqqq6O3tRXx8PI4cOYLMzEw0NTXBaDQiMTExqN5iscDlcgEAXC5XUPgMzg/O3YvP54PP59P3vV4vAEDTNF4/GqfudwseAGp/ugrg6rCPd+fOnfvepqfQ9PT0DKsu5AB6+umn0dTUBE3T8O2336KgoAB1dXUhNxiKkpIS7Nq1667x7OxsKIryUJ+bxqbR/jOXkZGBBQsWjOoxx7PBNwl/JeR/CW00GvHkk08iKysLJSUlmDVrFj7++GNYrVb4/f67/hZxu92wWq0AAKvVetddscH9wZqhFBcXQ9M0fevo6Ai1bSIagx74qxiBQAA+nw9ZWVmIjY1FTU2NPtfW1ob29naoqgoAUFUVzc3NQZ/fq6uroSgKMjMz7/kcJpNJv/U/uBFR5AvpI1hxcTGWLFkCm82Grq4uHD58GKdPn8bJkydhNpuxZs0aFBUVISkpCYqiYOPGjVBVFfPnzwcA5OTkIDMzE6tWrcLevXvhcrmwbds2OBwOmEymh3KCRDR2hRRAHo8Hb7zxBm7cuAGz2YyZM2fi5MmTePHFFwEA+/btg8FgQH5+Pnw+H3Jzc/HZZ5/pj4+OjkZVVRU2bNgAVVUxadIkFBQUYPfu3aN7VvTIG3xXPFpiY2NH7Vg0fFFCCCG7iVB5vV6YzWZomsaPY+OU1+uFpmmjdrzk5GTExcWN2vHGu+G+RvldMIpIvBb4aODPAyIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSxMhuYCSEEAAAr9cruRMiGsrga3PwtXovERlAt27dAgCkpaVJ7oSI7qerqwtms/me8xEZQElJSQCA9vb2+54cBfN6vUhLS0NHRwcURZHdTkTgmo2MEAJdXV1ITU29b11EBpDB8M9LV2azmX8oRkBRFK5biLhmoRvOmwNehCYiaRhARCRNRAaQyWTCzp07YTKZZLcSUbhuoeOaPVxR4q/ukxERPSQR+Q6IiB4NDCAikoYBRETSMICISJqIDKDS0lJMnToVEyZMQHZ2NhoaGmS3JE1JSQnmzp2LhIQEpKSkYPny5Whrawuq6e3thcPhQHJyMuLj45Gfnw+32x1U097ejry8PMTFxSElJQVbt25Ff39/OE9Fmj179iAqKgqbN2/Wx7hmYSIiTEVFhTAajeLLL78ULS0tYu3atSIxMVG43W7ZrUmRm5srysvLxaVLl0RTU5NYunSpsNlsoru7W69Zv369SEtLEzU1NeLChQti/vz5YsGCBfp8f3+/mD59urDb7eKnn34Sx44dE5MnTxbFxcUyTimsGhoaxNSpU8XMmTPFpk2b9HGuWXhEXADNmzdPOBwOfX9gYECkpqaKkpISiV2NHR6PRwAQdXV1QgghOjs7RWxsrKisrNRrLl++LAAIp9MphBDi2LFjwmAwCJfLpdeUlZUJRVGEz+cL7wmEUVdXl8jIyBDV1dXi+eef1wOIaxY+EfURzO/3o7GxEXa7XR8zGAyw2+1wOp0SOxs7NE0D8K8v7DY2NqKvry9ozaZNmwabzaavmdPpxIwZM2CxWPSa3NxceL1etLS0hLH78HI4HMjLywtaG4BrFk4R9WXUmzdvYmBgIOg3HQAsFguuXLkiqauxIxAIYPPmzVi4cCGmT58OAHC5XDAajUhMTAyqtVgscLlces1Qazo49yiqqKjAxYsXcf78+bvmuGbhE1EBRPfncDhw6dIl/Pjjj7JbGdM6OjqwadMmVFdXY8KECbLbGdci6iPY5MmTER0dfdfdCLfbDavVKqmrsaGwsBBVVVU4deoUpkyZoo9brVb4/X50dnYG1f9xzaxW65BrOjj3qGlsbITH48Fzzz2HmJgYxMTEoK6uDvv370dMTAwsFgvXLEwiKoCMRiOysrJQU1OjjwUCAdTU1EBVVYmdySOEQGFhIY4cOYLa2lqkp6cHzWdlZSE2NjZozdra2tDe3q6vmaqqaG5uhsfj0Wuqq6uhKAoyMzPDcyJhtHjxYjQ3N6OpqUnf5syZg5UrV+q/5pqFieyr4KGqqKgQJpNJHDx4ULS2top169aJxMTEoLsR48mGDRuE2WwWp0+fFjdu3NC3O3fu6DXr168XNptN1NbWigsXLghVVYWqqvr84C3lnJwc0dTUJE6cOCEef/zxcXVL+Y93wYTgmoVLxAWQEEJ88sknwmazCaPRKObNmyfq6+tltyQNgCG38vJyveb3338Xb731lnjsscdEXFyceOWVV8SNGzeCjvPrr7+KJUuWiIkTJ4rJkyeLt99+W/T19YX5bOT5cwBxzcKDP46DiKSJqGtARPRoYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETS/B++oTXmnqKVgAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
